import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from pathlib import Path
from datasets import Dataset

import langdetect
import re

from mpl_toolkits.axes_grid1.inset_locator import inset_axes

from collections import Counter
from collections import OrderedDict
from langdetect import detect

import torch

import sys
import os
import warnings
warnings.filterwarnings('ignore')
sys.path.append(os.path.abspath("../"))
import src.utilities as u
import src.plots as p
import src.constants as c
c_stopwords = c.custom_stopwords

from src.utilities import TextCleaner
from src.credentials import AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_openai import AzureOpenAIEmbeddings


from langchain.vectorstores import Chroma
from langchain.embeddings import OpenAIEmbeddings
import chromadb
from src.evaluation import RAGAS_Evaluator

from langchain_core.runnables import RunnablePassthrough
from langchain_core.output_parsers import StrOutputParser
from langchain_openai import ChatOpenAI
from langchain_openai import AzureChatOpenAI
from langchain_core.runnables import RunnableLambda
from langchain_core.documents import Document
from langchain.schema.runnable import RunnableMap

from langchain_core.prompts import ChatPromptTemplate


from ragas.metrics import faithfulness, answer_relevancy, context_precision
from ragas import evaluate
from datasets import Dataset
from difflib import SequenceMatcher

u.generate_toc('../doc/MC1.ipynb')

file_path = "../data_mc1/"
file_name_train = 'cleantech_media_dataset_v3_2024-10-28.csv'

column_names = ["id", "title", "date", "author", "content", "domain", "url"]
df_train = pd.read_csv(file_path+file_name_train, delimiter=",", names=column_names, skiprows=1, encoding="utf-8")
df_train.head()

u.missing_value_stats(df_train)

df_train = df_train.drop(columns=["author"])
df_train.dtypes

id          int64
title      object
date       object
content    object
domain     object
url        object
dtype: object

styled = (
    df_train.describe(include=[object])
    .applymap(u.shorten_text)  # auf alle Zellen anwenden
    .style
    .set_properties(**{
        'white-space': 'pre-wrap',    # \n sichtbar
        'max-width': '300px',         # Zellbreite beschränken
        'font-family': 'monospace',   # bessere Darstellung
    })
)

styled

# Datentypen korrigieren
df_train["date"] = pd.to_datetime(df_train["date"], errors="coerce")  # Falls ungültige Datumswerte vorhanden

df_train.dtypes

id                  int64
title              object
date       datetime64[ns]
content            object
domain             object
url                object
dtype: object

articles_per_domain = df_train.groupby("domain").size().reset_index(name="count")
articles_per_domain = articles_per_domain.sort_values(by="count", ascending=False)
articles_per_domain.reset_index(drop=True, inplace=True)
p.bar_plot(articles_per_domain, "domain", "count", "Anzahl Artikel pro Domain", "Domain", "Anzahl Artikel", xrotation=45, ha_xrotation='right')

articles_per_title_top_15 = df_train.groupby("title").size().reset_index(name="count")
articles_per_title_top_15 = articles_per_title_top_15.sort_values(by="count", ascending=False).head(15)
articles_per_title_top_15.reset_index(drop=True, inplace=True)
p.bar_plot(articles_per_title_top_15, "title", "count", "Anzahl Artikel pro Titel", "Titel", "Anzahl Artikel", xrotation=45, ha_xrotation='right', height_offset=0)

article_count_per_title = df_train[df_train["title"] == "About David J. Cross"].groupby("domain").size().reset_index(name="count")
article_count_per_title = article_count_per_title.sort_values(by="count", ascending=False)
article_count_per_title.reset_index(drop=True, inplace=True)
article_count_per_title

article_count_per_title = df_train[df_train["title"] == "About David J. Cross"]
u.styled_text(article_count_per_title)

# nach Datum gruppieren und Anzahl Artikel zählen, absteigend sortieren
articles_per_date = df_train.groupby("date").size().reset_index(name="count")
articles_per_date_sorted = articles_per_date.sort_values(by="count", ascending=False)
articles_per_date_sorted.head(10)

# Plot erstellen
plt.figure(figsize=(12, 6))
plt.plot(articles_per_date["date"], articles_per_date["count"], linestyle='-')
plt.title("Entwicklung über die Zeit")
plt.xlabel("Datum")
plt.ylabel("Wert")
plt.grid(True)
plt.tight_layout()
plt.xticks(rotation=45)
plt.show()

# Hauptplot
fig, ax = plt.subplots(figsize=(14, 6))
ax.plot(articles_per_date["date"], articles_per_date["count"], label="Count")
ax.set_title("Total Veröffentlichungen über die Zeit")
ax.set_xlabel("Datum")
ax.set_ylabel("Wert")
ax.grid(True)

# Inset-Plot (Zoom auf normalen Bereich)
axins = inset_axes(ax, width="45%", height="55%", loc="upper left", borderpad=2.5)
axins.plot(articles_per_date["date"], articles_per_date["count"])
axins.set_ylim(0, articles_per_date["count"].quantile(0.95))  # 95%-Perzentil als Cutoff
axins.set_title("Zoom (unter 95. Perzentil)")
axins.tick_params(labelsize=8)

plt.tight_layout()
plt.show()

article_per_date_and_domain = df_train[df_train["date"] == "2023-06-13"].groupby("domain").size().reset_index(name="count")
article_per_date_and_domain = article_per_date_and_domain.sort_values(by="count", ascending=False)
article_per_date_and_domain.reset_index(drop=True, inplace=True)
article_per_date_and_domain

energy_xprt_title_count = df_train[
    (df_train["date"] == "2023-06-13") & (df_train["domain"] == "energy-xprt")
].groupby("title").size().reset_index(name="count")

energy_xprt_title_count = energy_xprt_title_count.sort_values(by="count", ascending=False)
energy_xprt_title_count.reset_index(drop=True, inplace=True)
energy_xprt_title_count

content = " ".join(df_train["content"].astype(str).tolist())
p.plot_wordcloud(content, c_stopwords=c_stopwords, column_name='content')

titles = " ".join(df_train["title"].astype(str).tolist())
p.plot_wordcloud(titles, c_stopwords=c_stopwords, column_name='title')

def check_language(df: pd.DataFrame, column: str) -> bool:
    """
    Prüft, ob alle Texte in einer bestimmten Spalte auf Englisch sind.

    Args:
        df
        column: Name der Spalte mit Text.

    Returns:
        bool: True, wenn alle Inhalte in der Spalte Englisch sind, sonst False.
    """
    try:
        all_english = all(
            langdetect.detect(text) == "en"
            if isinstance(text, str) and text.strip() != "" else False
            for text in df[column]
        )
    
        if all_english:
            print("Alle Inhalte sind auf Englisch.")
        else:
            print("Die Sprache ist nicht einheitlich Englisch.")
    
    except Exception as e:
        print(f"Fehler bei Spracherkennung: {e}")
        return False
    

check_language(df_train, "content")

Die Sprache ist nicht einheitlich Englisch.

# Sprache für jede Zeile erkennen
def detect_language(text: str) -> str:
    try:
        return detect(text) if isinstance(text, str) and text.strip() != "" else "unknown"
    except:
        return "error"

# Neue Spalte Spracherkennung
df_train["language"] = df_train["content"].apply(detect_language)

# Sprachen zählen
language_counts = Counter(df_train["language"])
print("Anzahl Artikel pro erkannter Sprache:")
print(language_counts)

Anzahl Artikel pro erkannter Sprache:
Counter({'en': 20106, 'de': 3, 'ru': 1, 'es': 1})

u.print_full_text(df_train[df_train["language"] == 'de'][['id', 'title', 'content', 'language']].head(1), max_width=400)

# Nur englische Artikel behalten
df_train = df_train[df_train["language"] == "en"]

print(f"Anzahl Artikel nach dem Entfernen nicht-englischer Inhalte: {len(df_train)}")

Anzahl Artikel nach dem Entfernen nicht-englischer Inhalte: 20106

def fake_list_to_text(text: str) -> str:
    if not isinstance(text, str):
        return text
    if text.count("',") >= 1 and "'" in text:
        parts = re.split(r"\s*,\s*", text.strip("[]"))
        parts = [p.strip(" '\"\n") for p in parts if p.strip()]
        return " ".join(parts)
    return text

df_train["content"] = df_train["content"].apply(fake_list_to_text)
df_train.head()

cleaner = TextCleaner(level="risky")
data_cleaned = cleaner.clean_text_column(article_count_per_title, column="content", new_column="content_cleaned")

data_cleaned_compare = data_cleaned[["content", "content_cleaned"]].copy()
u.styled_text(data_cleaned_compare.head(), max_width=500)

df_train_cleaned = cleaner.clean_text_column(df_train, column="content", new_column="content_cleaned")

u.styled_text(df_train_cleaned.head(3))

u.styled_text(df_train_cleaned.describe(include=[object]))

def drop_duplicates_keep_latest(df: pd.DataFrame) -> pd.DataFrame:
    """
    Entfernt Duplikate basierend auf den Spalten 'title', 'content_cleaned', 'domain', und 'url' und behält den neuesten Eintrag.
    """
    df['date'] = pd.to_datetime(df['date'], errors='coerce')
    df = df.sort_values('date').drop_duplicates(subset=['content_cleaned'], keep='last')
    
    return df
df_train_cleaned_unique = df_train_cleaned.copy(deep=True)
df_train_cleaned_unique = drop_duplicates_keep_latest(df_train_cleaned_unique)
len(df_train_cleaned), len(df_train_cleaned_unique), len(df_train_cleaned) - len(df_train_cleaned_unique)

(20106, 19988, 118)

def check_target_url_in_train_df(target_df: pd.DataFrame, train_df: pd.DataFrame) -> pd.DataFrame:
    url_in_train_df = set(train_df['url'])
    target_df['url_in_train_df'] = target_df['article_url'].isin(url_in_train_df)
    return target_df
    
# read parquet file
cleantech_rag_evaluation_data = pd.read_parquet('../data_mc1/data_processed/cleantech_rag_evaluation_data.parquet')
url_check = check_target_url_in_train_df(cleantech_rag_evaluation_data, df_train_cleaned_unique)
print(f"Es fehlen {len(url_check[url_check['url_in_train_df'] == False])} Datensätze, welch durch die Bereinigung gelöscht wurden und im  Evaluationsdatensatz enthalten sind.")

Es fehlen 0 Datensätze, welch durch die Bereinigung gelöscht wurden und im  Evaluationsdatensatz enthalten sind.

print(f"Nach dem Bereinigen sind noch {len(df_train_cleaned_unique)} Artikel übrig (von ursprünglich {len(df_train)}), was einer Reduktion von {len(df_train) - len(df_train_cleaned_unique)} Artikeln entspricht.")
df_train_cleaned = df_train_cleaned_unique.copy(deep=True)

Nach dem Bereinigen sind noch 19988 Artikel übrig (von ursprünglich 20106), was einer Reduktion von 118 Artikeln entspricht.

content = " ".join(df_train_cleaned["content_cleaned"].astype(str).tolist())
p.plot_wordcloud(content, c_stopwords=c_stopwords, column_name='content_cleaned')

titles = " ".join(df_train_cleaned["title"].astype(str).tolist())
p.plot_wordcloud(titles, c_stopwords=c_stopwords, column_name='title')

df_train_cleaned = df_train_cleaned.drop(columns=["content"])
df_train_cleaned = df_train_cleaned.rename(columns={"content_cleaned": "content"})

# ermittle die maximale Länge der Titel
max_title_length = df_train_cleaned['title'].str.len().max()
print(f"Maximale Länge der Titel: {max_title_length}")
# ermittle die minimale Länge der Inhalte
min_content_length = df_train_cleaned['content'].str.len().min()
print(f"Minimale Länge der Inhalte: {min_content_length}")

# ermittle, welche Titel länger sind als ihre Inhalte
longer_titles = df_train_cleaned[df_train_cleaned['title'].str.len() > df_train_cleaned['content'].str.len()]
print(f"Anzahl Titel, die länger sind als ihre Inhalte: {len(longer_titles)}")

Maximale Länge der Titel: 403
Minimale Länge der Inhalte: 8
Anzahl Titel, die länger sind als ihre Inhalte: 36

# Kontrolliere, ob die evaluationadaten (urls) in den longer_titles enthalten sind
print(f"Anzahl Evaluationsdaten, die in den longer_titles enthalten sind: {len(longer_titles[longer_titles['url'].isin(cleantech_rag_evaluation_data['article_url'])])}")

Anzahl Evaluationsdaten, die in den longer_titles enthalten sind: 0

# Lösche die Datensätze von longer_titles aus dem df_train_cleaned
df_train_cleaned = df_train_cleaned[~df_train_cleaned['id'].isin(longer_titles['id'])]

# Title und Content zusammenführen
df_train_cleaned['text'] = df_train_cleaned['title'] + ": " + df_train_cleaned['content']
df_train_cleaned.head(3)

df_train_cleaned = df_train_cleaned.drop(columns=["content"])
df_train_cleaned = df_train_cleaned.rename(columns={"text": "content"})

df_train_cleaned_subset = df_train_cleaned[df_train_cleaned['url'].isin(cleantech_rag_evaluation_data['article_url'])].copy(deep=True)
df_train_subset_1000 = df_train_cleaned.sample(n=1000-len(df_train_cleaned_subset), random_state=42, replace=False)
df_train_1000 =pd.concat([df_train_cleaned_subset, df_train_subset_1000], ignore_index=True)

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-3-large")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_recchar_800_cosine_3-large",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="800_cosine_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:15<00:00,  1.25it/s]

Durchschnittliche Antwortähnlichkeit: 0.8089461937183137

evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-ada-002")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_recchar_800_cosine_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="ada_800_cosine_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:15<00:00,  1.26it/s]

Durchschnittliche Antwortähnlichkeit: 0.9224537753013365

evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-3-large")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_recchar_800_ip_3-large",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="800_ip_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:20<00:00,  1.18it/s]

Durchschnittliche Antwortähnlichkeit: 0.7402009444554254

evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-ada-002")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_recchar_800_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="ada_800_ip_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:18<00:00,  1.21it/s]

Durchschnittliche Antwortähnlichkeit: 0.9300966413118155

evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-3-large")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_recchar_800_l2_3-large",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="800_l2_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:19<00:00,  1.20it/s]

Durchschnittliche Antwortähnlichkeit: 0.7679850248820453

evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-ada-002")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_recchar_800_l2_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="ada_800_l2_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:23<00:00,  1.13it/s]

Durchschnittliche Antwortähnlichkeit: 0.9318871484441911

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-3-large")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_cosine_3-large",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="200_cosine_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:27<00:00,  1.08it/s]

Durchschnittliche Antwortähnlichkeit: 0.8183913191958395

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-ada-002")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_cosine_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="ada_200_cosine_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:39<00:00,  1.05s/it]

Durchschnittliche Antwortähnlichkeit: 0.9421745395976444

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-3-large")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_3-large",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="200_ip_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:41<00:00,  1.07s/it]

Durchschnittliche Antwortähnlichkeit: 0.8382218615881519

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-ada-002")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="ada_200_ip_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:40<00:00,  1.06s/it]

Durchschnittliche Antwortähnlichkeit: 0.9428049017660128

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-3-large")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_l2_3-large",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="200_l2_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:42<00:00,  1.08s/it]

Durchschnittliche Antwortähnlichkeit: 0.8280307023708715

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-ada-002")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_l2_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="ada_200_l2_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:39<00:00,  1.05s/it]

Durchschnittliche Antwortähnlichkeit: 0.9413381270921989

# Ausführung des Codes
evaluator = RAGAS_Evaluator(embedding_deployment="text-embedding-ada-002")

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="chain_of_thought",
    temperature=0.3,
    save_path="sim_k5_cot.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:17<00:00,  1.23it/s]

Durchschnittliche Antwortähnlichkeit: 0.8449529152214904

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k3",
    prompt_name="simple",
    temperature=0.3,
    save_path="sim_k3_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:14<00:00,  1.27it/s]

Durchschnittliche Antwortähnlichkeit: 0.9366147935818292

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k3",
    prompt_name="chain_of_thought",
    temperature=0.3,
    save_path="sim_k3_cot.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:15<00:00,  1.26it/s]

Durchschnittliche Antwortähnlichkeit: 0.8406827692445922

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="mmr_k5",
    prompt_name="simple",
    temperature=0.3,
    save_path="mmr_k5_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:34<00:00,  1.00it/s]

Durchschnittliche Antwortähnlichkeit: 0.896906079895636

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="mmr_k5",
    prompt_name="chain_of_thought",
    temperature=0.3,
    save_path="mmr_k5_cot.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:16<00:00,  1.24it/s]

Durchschnittliche Antwortähnlichkeit: 0.8322374389077946

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="mmr_k7",
    prompt_name="simple",
    temperature=0.3,
    save_path="mmr_k7_simple.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [02:13<00:00,  1.41s/it]

Durchschnittliche Antwortähnlichkeit: 0.90163346467248

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="mmr_k7",
    prompt_name="chain_of_thought",
    temperature=0.3,
    save_path="mmr7_k7_cot.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [02:09<00:00,  1.36s/it]

Durchschnittliche Antwortähnlichkeit: 0.8361902117960175

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.0,
    save_path="temp_0.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating:  97%|█████████▋| 92/95 [01:36<00:11,  3.69s/it]ERROR:ragas.executor:Exception raised in Job[0]: TimeoutError()
Evaluating:  98%|█████████▊| 93/95 [03:00<00:53, 26.77s/it]ERROR:ragas.executor:Exception raised in Job[4]: TimeoutError()
ERROR:ragas.executor:Exception raised in Job[40]: TimeoutError()
Evaluating: 100%|██████████| 95/95 [03:09<00:00,  2.00s/it]

Durchschnittliche Antwortähnlichkeit: 0.9419080395988685

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.5,
    save_path="temp_05.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:41<00:00,  1.07s/it]

Durchschnittliche Antwortähnlichkeit: 0.944691514754571

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=0.7,
    save_path="temp_07.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:41<00:00,  1.07s/it]

Durchschnittliche Antwortähnlichkeit: 0.9384527929291889

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=1.0,
    save_path="temp_1.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:39<00:00,  1.05s/it]

Durchschnittliche Antwortähnlichkeit: 0.924666588835609

results, ragas_metrics, ir_metrics = evaluator.run(
    collection_name="chroma_senttrans_200_ip_ada-002",
    retriever_name="similarity_k5",
    prompt_name="simple",
    temperature=2.0,
    save_path="temp_2.csv",
    plot=True
)

# Berechne die Antwortähnlichkeit
results = evaluator.compute_similarity_to_gold(results)
print("Durchschnittliche Antwortähnlichkeit:", results["similarity_to_gold"].mean())

Evaluating: 100%|██████████| 95/95 [01:14<00:00,  1.28it/s]

Durchschnittliche Antwortähnlichkeit: 0.9094658910848308

df_manuell_samples = pd.read_csv("../data_mc1/data_evaluierung/temp_05.csv")


for i, row in df_manuell_samples.iloc[:4, :6].iterrows():
    print(f"\n--- Sample {i} ---")
    for col in df_manuell_samples.columns[:6]:
        print(f"{col}:\n{row[col]}\n")

--- Sample 0 ---
question:
What is the EU’s Green Deal Industrial Plan?

relevant_text:
The European counterpart to the US Inflation Reduction Act (IRA) aims to create an environment that is conducive to increasing the European Union's manufacturing capacity of “net-zero” technologies, to improve the competitiveness of European industry.

relevant_text_llm:
['european commission introduces green deal industrial plan – pv magazine international : the european commission listed tax exemptions flexible aid and the promotion of local manufacturing among its key strategies for the green deal industrial plan. however the solarpower europe industry group has criticized the plan \' s lack of focus on solar. european commission president ursula von der leyen presents the green deal industrial plan. the european commission has published a new document on the green deal industrial plan that ec president ursula von der leyen announced at davos in mid january. the european counterpart to the us inflation reduction act ira aims to create an environment that is conducive to increasing the european union \' s manufacturing capacity of " net zero " technologies to improve the competitiveness of european industry. the plan proposed this week has yet to be discussed among member states and will be presented at the european summit in march. it includes proposals to make state aid more flexible mobilize european funds create a sovereign wealth fund and improve european supply chains. the state aid revisions would apply until 2025 in comparison with the ira \' s 10 year time frame. the industrial plan will be based on four pillars a predictable and simplified regulatory environment better access to finance upskilling and open trade for resilient supply chains. as part of the first pillar the', 'europe \' s green transition and energy security. dries acke policy director at solarpower europe said in a statement last week " industrial policy focus on critical renewable energy supply chains is a good thing. however we \' re worried by the lack of focus in the green deal industrial plan. we need support specifically for the bulk technologies that net zero and energy security are based on. " a more detailed extent of the plan still remains to be seen and some uncertainty as to how companies could be eligible for funding or the extent of such funding is yet to be unveiled. the ira contains a detailed list of manufacturing credits throughout the value chain to boost the us pv manufacturing industry. " we certainly welcome eu initiatives such as the green deal industrial plan. however it will be of crucial importance how exactly such initiatives will be implemented. precise details on the actions eu commission intends to take remain to be seen " says a spokesperson at polysilicon manufacturer wacker. so far one of the major differences between europe \' s funding and the ira is how manufacturers get access to it and how long the process can be says davor sutija ceo at wafer producer nexwafe. in europe the application and funding can take up to two years while in the us it will be dependent on the company', 'european solar manufacturers welcome green deal industrial plan : last week ursula von der leyen president of the european commission ec unveiled the eu \' s response to the inflation reduction act ira and china \' s dominance in the solar supply chain with the green deal industrial plan gdip which will be based on four pillars a predictable and simplified regulatory environment faster access to funding enhancing skills and open trade for resilient supply chains. if the ira has been welcomed by the us solar industry almost as the saviour to boost a domestic manufacturing industry in europe the response seems more lukewarm. manufacturers from across the value chain have welcomed the initiative yet are asking for more if the eu expects to compete on a level playing field with the likes of china which currently has a share of more than 80 of the global value chain the us and india. " the gdip is a very good first step but it has to be followed by others soon a strategic alliance of capital business and manufacturers and a plan for how to launch the industry " says udo mohrstedt ceo at module manufacturer ibc solar. trade body solarpower europe is calling on the eu to support technologies such as solar pv for europe \' s green transition and energy security. dries acke policy director at solarpower europe said', 'support the green transition. it will also explore the creation of a " critical raw materials club " to bring raw material consumers and resource rich countries together with the aim of ensuring global security of supply through a competitive diversified industrial base. the industrial plan is short on more details and does not discriminate between " net zero " technologies in its policy proposals. this is something that solarpower europe has criticized. " industrial policy focus on critical renewable energy supply chains is a good thing. however we \' re worried by the lack of focus in today \' s communication " said dries acke policy director at solarpower europe. " not all net zero technologies are in the same boat not in terms of strategic importance or even the impact they \' re feeling from the inflation reduction act. " ursula von der leyen said during the presentation of the plan that " europe is determined to lead the clean technology revolution. " she added that better access to financing will allow key industries of clean technologies to " grow rapidly. " brussels has its sights set on the extraordinary summit of eu leaders on feb. 9 and feb. 10. the meeting is expected to address the community response to subsidies from the united states and from china where announced investments in clean technologies will exceed 280 billion.', 'the 40 domestic production benchmark. " the act will enable the development in europe of projects in key sectors such as batteries solar cells hydrogen and wind turbines as well as all other projects in the connected value chains. this will help us deliver on the green deal objectives whilst ensuring a level playing field " said margrethe vestager executive vice president for a europe fit for the digital age. at the same time the ec has announced the critical raw materials act to diversify its imports of critical raw materials. it admitted that europe heavily relies on imports which are often from \' quasi monopolistic third country suppliers \' and will not be self sufficient in supplying such raw materials. therefore the eu will diversify its raw material supply chain with no more than 65 of its annual consumption of each raw material from a single third country and seek \' mutually beneficial \' partnerships with emerging']

retrieved_context:
european commission introduces green deal industrial plan – pv magazine international : the european commission listed tax exemptions flexible aid and the promotion of local manufacturing among its key strategies for the green deal industrial plan. however the solarpower europe industry group has criticized the plan ' s lack of focus on solar. european commission president ursula von der leyen presents the green deal industrial plan. the european commission has published a new document on the green deal industrial plan that ec president ursula von der leyen announced at davos in mid january. the european counterpart to the us inflation reduction act ira aims to create an environment that is conducive to increasing the european union ' s manufacturing capacity of " net zero " technologies to improve the competitiveness of european industry. the plan proposed this week has yet to be discussed among member states and will be presented at the european summit in march. it includes proposals to make state aid more flexible mobilize european funds create a sovereign wealth fund and improve european supply chains. the state aid revisions would apply until 2025 in comparison with the ira ' s 10 year time frame. the industrial plan will be based on four pillars a predictable and simplified regulatory environment better access to finance upskilling and open trade for resilient supply chains. as part of the first pillar the

europe ' s green transition and energy security. dries acke policy director at solarpower europe said in a statement last week " industrial policy focus on critical renewable energy supply chains is a good thing. however we ' re worried by the lack of focus in the green deal industrial plan. we need support specifically for the bulk technologies that net zero and energy security are based on. " a more detailed extent of the plan still remains to be seen and some uncertainty as to how companies could be eligible for funding or the extent of such funding is yet to be unveiled. the ira contains a detailed list of manufacturing credits throughout the value chain to boost the us pv manufacturing industry. " we certainly welcome eu initiatives such as the green deal industrial plan. however it will be of crucial importance how exactly such initiatives will be implemented. precise details on the actions eu commission intends to take remain to be seen " says a spokesperson at polysilicon manufacturer wacker. so far one of the major differences between europe ' s funding and the ira is how manufacturers get access to it and how long the process can be says davor sutija ceo at wafer producer nexwafe. in europe the application and funding can take up to two years while in the us it will be dependent on the company

european solar manufacturers welcome green deal industrial plan : last week ursula von der leyen president of the european commission ec unveiled the eu ' s response to the inflation reduction act ira and china ' s dominance in the solar supply chain with the green deal industrial plan gdip which will be based on four pillars a predictable and simplified regulatory environment faster access to funding enhancing skills and open trade for resilient supply chains. if the ira has been welcomed by the us solar industry almost as the saviour to boost a domestic manufacturing industry in europe the response seems more lukewarm. manufacturers from across the value chain have welcomed the initiative yet are asking for more if the eu expects to compete on a level playing field with the likes of china which currently has a share of more than 80 of the global value chain the us and india. " the gdip is a very good first step but it has to be followed by others soon a strategic alliance of capital business and manufacturers and a plan for how to launch the industry " says udo mohrstedt ceo at module manufacturer ibc solar. trade body solarpower europe is calling on the eu to support technologies such as solar pv for europe ' s green transition and energy security. dries acke policy director at solarpower europe said

support the green transition. it will also explore the creation of a " critical raw materials club " to bring raw material consumers and resource rich countries together with the aim of ensuring global security of supply through a competitive diversified industrial base. the industrial plan is short on more details and does not discriminate between " net zero " technologies in its policy proposals. this is something that solarpower europe has criticized. " industrial policy focus on critical renewable energy supply chains is a good thing. however we ' re worried by the lack of focus in today ' s communication " said dries acke policy director at solarpower europe. " not all net zero technologies are in the same boat not in terms of strategic importance or even the impact they ' re feeling from the inflation reduction act. " ursula von der leyen said during the presentation of the plan that " europe is determined to lead the clean technology revolution. " she added that better access to financing will allow key industries of clean technologies to " grow rapidly. " brussels has its sights set on the extraordinary summit of eu leaders on feb. 9 and feb. 10. the meeting is expected to address the community response to subsidies from the united states and from china where announced investments in clean technologies will exceed 280 billion.

the 40 domestic production benchmark. " the act will enable the development in europe of projects in key sectors such as batteries solar cells hydrogen and wind turbines as well as all other projects in the connected value chains. this will help us deliver on the green deal objectives whilst ensuring a level playing field " said margrethe vestager executive vice president for a europe fit for the digital age. at the same time the ec has announced the critical raw materials act to diversify its imports of critical raw materials. it admitted that europe heavily relies on imports which are often from ' quasi monopolistic third country suppliers ' and will not be self sufficient in supplying such raw materials. therefore the eu will diversify its raw material supply chain with no more than 65 of its annual consumption of each raw material from a single third country and seek ' mutually beneficial ' partnerships with emerging

answer:
The EU’s Green Deal Industrial Plan aims to enhance the competitiveness of its net zero industry and accelerate the transition to net zero by supporting the expansion of European manufacturing of technologies, goods, and services necessary to meet climate targets.

answer_llm:
The EU’s Green Deal Industrial Plan is a strategy introduced by the European Commission to enhance the manufacturing capacity of "net zero" technologies, improve the competitiveness of European industry, and address challenges posed by the US Inflation Reduction Act and China's dominance in the solar supply chain.


--- Sample 1 ---
question:
When did the cooperation between GM and Honda on fuel cell vehicles start?

relevant_text:
What caught our eye was a new hookup between GM and Honda. Honda was also hammering away at the challenge of introducing fuel cell vehicles to the US market. The collaboration launched in July of 2013, providing for the two companies to share the combined total of 1,200 fuel cell patents they compiled in the years leading up to 2012.

relevant_text_llm:
['general motors seizes fuel cell moment with green hydrogen : green hydrogen is needed to ice the zero emission fuel cell electric cake for gm \' s silent utility rover universal superstructure vehicle platform. general motors has dropped hints about its interest in fuel cell electric vehicles over the years but apparently that is just the tip of a much bigger iceberg. the company has made some interesting moves in recent months which suggest a more holistic approach to the green hydrogen economy of the future regardless of what the skeptics may say. gm \' s interest in hydrogen caught the cleantechnica eye back in 2013 after the company introduced a demonstration fleet of 16 fuel cell electric vehicles for the us army in hawaii. the army demonstration was built on gm \' s project driveway fleet of 119 vehicles which launched in 2007 and racked up almost 3 million miles of driving in six years. what caught our eye was a new hookup between gm and honda. honda was also hammering away at the challenge of introducing fuel cell vehicles to the us market. the collaboration launched in july of 2013 providing for the two companies to share the combined total of 1 200 fuel cell patents they compiled in the years leading up to 2012. gm introduced its " surus " fuel cell electric vehicle platform in 2017 but it still didn \' t have a', 'its " surus " fuel cell electric vehicle platform in 2017 but it still didn \' t have a fuel cell ev in commercial production by 2020 though a preliminary agreement with nikola motor involving the badger fuel cell electric pickup truck suggested that activity was about to pick up. unfortunately for fcev fans nikola fell on its face following a scandal involving former ceo trevor milton. nikola has since recovered itself but the relationship with gm foundered. gm has not been idling around since then. as part of its collaboration with honda gm set its sights on developing fuel cells for heavy duty use resulting in its hydrotec suite of fuel cell systems. in 2021 gm launched a hydrogen powered flight collaboration with hydrotec and the aerospace company liebnerr. gm is also collaborating with navistar on fuel cells for trucks and with wabtec on fuel cells for locomotives. gm has also been deploying hydrotec modules as transportable charging stations for battery electric vehicles. the sticky wicket of course is the continued reliance on natural gas and other fossil resources to produce the hydrogen needed to run fuel cells. gm has a plan for that too. last november gm enlisted nel usa a unit of norway \' s nel asa to help push down the cost of electrolyzer systems and pump up the green hydrogen supply chain', 'hitch in gm \' s brightdrop battery electric delivery van production plans when its factory in canada ran short on batteries. gm is among the us automakers continuing to hold the fuel cell torch a position that the battery shortage appears to vindicate. also worth noting is the fuel cell startup nikola motors which appears to be back on its feet after stumbling a couple of years ago. if all goes according to plan nikola will bring its zero emission fuel cell trucks to the us postal service among other customers. meanwhile in may ford announced that its popular e transit delivery van will be the platform for a three year hydrogen fuel cell research project funded in part by the uk \' s advanced propulsion center. " the project will establish if hydrogen fuel cell technology can deliver more zero emission range to heavy use e transit customers travelling high mileages with maximum loads ancillary equipment such as chillers and with limited charging opportunities in the working shift " ford explained. the program involves a fleet of eight fuel cell ford e transit vans which are assembled in turkey for the european market. if the new tata ev battery plant works out it \' s possible that a fuel cell factory could be in the works eventually. back in 2021 jaguar land rover announced that it is developing a fuel cell version of the defender in the uk with an', 'cell vehicle platform courtesy of gm hydrotec. tina specializes in military and corporate sustainability advanced technology emerging materials biofuels and water and wastewater issues. views expressed are her own. follow her on twitter tinamcasey and spoutible. a new cargo ship will use 100 wind energy to ferry goods between europe and the us without the carbon baggage. the us state of north dakota is among those not waiting around for the green hydrogen grass to grow under their feet. large or small heat pumps are a key pathway for rapid decarbonization. gm ceo mary barra suggested in a radio interview this week that there might be a next generation chevy bolt in the works..', 'jaguar land rover announced that it is developing a fuel cell version of the defender in the uk with an assist from the advanced propulsion center under a program called project zeus. tata is also getting a fuel cell assist from the government of india under the technology development and demonstration program in collaboration with the indian space research organization. last year the company showcased a fuel cell electric bus and in april it stepped up its ongoing partnership with the us firm cummins to bring more zero emission fuel cell vehicles to india. as for the us fuel cells are slowly trickling in but they have a lot of catching up to do. despite ongoing supply chain concerns plans to increase ev battery manufacturing capacity have been going through the roof in the us thanks in part to the one two punch of government incentives and low cost labor. find me on threads tinamcasey. also post tinamcasey or tinamcasey on linkedin and spoutible or casey on mastadon. image tata to build a new ev battery factory in the uk to support its ziptron electric vehicle technology image courtesy of tata motors. tina specializes in military and corporate sustainability advanced technology emerging materials biofuels and water and wastewater issues. views expressed are her own. follow her on twitter tinamcasey and spoutible.']

retrieved_context:
general motors seizes fuel cell moment with green hydrogen : green hydrogen is needed to ice the zero emission fuel cell electric cake for gm ' s silent utility rover universal superstructure vehicle platform. general motors has dropped hints about its interest in fuel cell electric vehicles over the years but apparently that is just the tip of a much bigger iceberg. the company has made some interesting moves in recent months which suggest a more holistic approach to the green hydrogen economy of the future regardless of what the skeptics may say. gm ' s interest in hydrogen caught the cleantechnica eye back in 2013 after the company introduced a demonstration fleet of 16 fuel cell electric vehicles for the us army in hawaii. the army demonstration was built on gm ' s project driveway fleet of 119 vehicles which launched in 2007 and racked up almost 3 million miles of driving in six years. what caught our eye was a new hookup between gm and honda. honda was also hammering away at the challenge of introducing fuel cell vehicles to the us market. the collaboration launched in july of 2013 providing for the two companies to share the combined total of 1 200 fuel cell patents they compiled in the years leading up to 2012. gm introduced its " surus " fuel cell electric vehicle platform in 2017 but it still didn ' t have a

its " surus " fuel cell electric vehicle platform in 2017 but it still didn ' t have a fuel cell ev in commercial production by 2020 though a preliminary agreement with nikola motor involving the badger fuel cell electric pickup truck suggested that activity was about to pick up. unfortunately for fcev fans nikola fell on its face following a scandal involving former ceo trevor milton. nikola has since recovered itself but the relationship with gm foundered. gm has not been idling around since then. as part of its collaboration with honda gm set its sights on developing fuel cells for heavy duty use resulting in its hydrotec suite of fuel cell systems. in 2021 gm launched a hydrogen powered flight collaboration with hydrotec and the aerospace company liebnerr. gm is also collaborating with navistar on fuel cells for trucks and with wabtec on fuel cells for locomotives. gm has also been deploying hydrotec modules as transportable charging stations for battery electric vehicles. the sticky wicket of course is the continued reliance on natural gas and other fossil resources to produce the hydrogen needed to run fuel cells. gm has a plan for that too. last november gm enlisted nel usa a unit of norway ' s nel asa to help push down the cost of electrolyzer systems and pump up the green hydrogen supply chain

hitch in gm ' s brightdrop battery electric delivery van production plans when its factory in canada ran short on batteries. gm is among the us automakers continuing to hold the fuel cell torch a position that the battery shortage appears to vindicate. also worth noting is the fuel cell startup nikola motors which appears to be back on its feet after stumbling a couple of years ago. if all goes according to plan nikola will bring its zero emission fuel cell trucks to the us postal service among other customers. meanwhile in may ford announced that its popular e transit delivery van will be the platform for a three year hydrogen fuel cell research project funded in part by the uk ' s advanced propulsion center. " the project will establish if hydrogen fuel cell technology can deliver more zero emission range to heavy use e transit customers travelling high mileages with maximum loads ancillary equipment such as chillers and with limited charging opportunities in the working shift " ford explained. the program involves a fleet of eight fuel cell ford e transit vans which are assembled in turkey for the european market. if the new tata ev battery plant works out it ' s possible that a fuel cell factory could be in the works eventually. back in 2021 jaguar land rover announced that it is developing a fuel cell version of the defender in the uk with an

cell vehicle platform courtesy of gm hydrotec. tina specializes in military and corporate sustainability advanced technology emerging materials biofuels and water and wastewater issues. views expressed are her own. follow her on twitter tinamcasey and spoutible. a new cargo ship will use 100 wind energy to ferry goods between europe and the us without the carbon baggage. the us state of north dakota is among those not waiting around for the green hydrogen grass to grow under their feet. large or small heat pumps are a key pathway for rapid decarbonization. gm ceo mary barra suggested in a radio interview this week that there might be a next generation chevy bolt in the works..

jaguar land rover announced that it is developing a fuel cell version of the defender in the uk with an assist from the advanced propulsion center under a program called project zeus. tata is also getting a fuel cell assist from the government of india under the technology development and demonstration program in collaboration with the indian space research organization. last year the company showcased a fuel cell electric bus and in april it stepped up its ongoing partnership with the us firm cummins to bring more zero emission fuel cell vehicles to india. as for the us fuel cells are slowly trickling in but they have a lot of catching up to do. despite ongoing supply chain concerns plans to increase ev battery manufacturing capacity have been going through the roof in the us thanks in part to the one two punch of government incentives and low cost labor. find me on threads tinamcasey. also post tinamcasey or tinamcasey on linkedin and spoutible or casey on mastadon. image tata to build a new ev battery factory in the uk to support its ziptron electric vehicle technology image courtesy of tata motors. tina specializes in military and corporate sustainability advanced technology emerging materials biofuels and water and wastewater issues. views expressed are her own. follow her on twitter tinamcasey and spoutible.

answer:
July 2013

answer_llm:
July 2013


--- Sample 2 ---
question:
Did Colgate-Palmolive enter into PPA agreements with solar developers?

relevant_text:
Scout Clean Energy, a Colorado-based renewable energy developer, owner and operator, has signed a power purchase agreement with Colgate-Palmolive for power from Scout’s 209 MW DC Markum Solar Farm near Waco, Texas.

relevant_text_llm:
['scout, colgate - palmolive sign ppa for texas solar farm : scout clean energy a colorado based renewable energy developer owner and operator has signed a power purchase agreement with colgate palmolive for power from scout \' s 209 mw dc markum solar farm near waco texas. this fall scout will begin and manage project construction. markum will generate enough clean energy for the ercot grid to power approximately 32 200 homes. " virtual power purchase agreements vppas are a valuable part of our renewable energy master plan and will help us achieve our targets of net zero carbon emissions by 2040 and 100 renewable electricity across our global operations by 2030 " says ann tracy colgate palmolive chief sustainability officer in an announcement. markum is in both mclennan and bosque counties and is scheduled for delivery in 2024. the project will result in approximately 300 temporary construction jobs and three permanent full time positions when the facility begins operation. economic benefits to the community from new tax revenue over the life of the project are expected to be significant. scout clean energy is a portfolio company of brookfield renewable a publicly traded pure play renewable power platform. brookfield acquired scout at the beginning of 2023 and scout continues to execute on its significant growth plans. scout \' s legal advisor', 'agreement will help us power our community with an additional cost effective investment in clean and renewable geothermal energy that aligns with our dedication to environmentally responsible generation resources " said svp chief utility officer manuel pineda. in 2022 calpine signed a similar 100 mw ppa with the sacramento municipality utility district smud for 10 years. this ppa was scheduled to take effect on january 1 2023.', "america signed a ppa with nestle for its under construction ganado solar plus storage project in jackson county in the south east of the state and projects such as these will help the company realise its solar power goals. the firm already manages 3. 3gw of installed wind and solar capacity in texas and plans to add another 1. 8gw of solar capacity in the coming years through projects in its pipeline. the news follows a string of new manufacturing announcements in the us such as trina solar ' s decision to build a new 5gw module manufacturing facility in the country. newfound enthusiasm for domestic manufacturing could help minimise the costs of building new solar facilities in the us although it remains to be seen if this expanded manufacturing capacity will translate to growth in power generation capacity.", 'mention soaring energy costs ppas are becoming an increasingly attractive option for businesses looking to secure a clean energy supply. greek energy company mytilineos recently signed two long term ppas with keppel dc reit to supply two dublin based data centres with energy generated from irish solar farms. with their high electricity demand data centres are ideal candidates for clean energy ppas. meanwhile solar developers are increasingly seeking novel and significant finance options to develop or expand solar projects. british solar renewables bsr has recently succeeded in this arena having secured its first portfolio financing thanks to a facility from commonwealth bank of australia and lloyds bank. the newly funded portfolio comprises five solar assets in the uk and australia with a combined annual solar output of 132gwh. thank you for subscribing to the newsletter. you can opt out at any time.', 'walmart inks solar pv ppa in texas, invests in community solar across us : retail giant walmart has signed a power purchase agreement ppa with utility edp renewables north america edpr na for a 180mw solar pv project in texas us. under a 15 year ppa walmart will offtake 162mw of the pv plant and complement existing corporate ppa between the companies on 233mw of utility scale solar and wind energy as well as 36mw of distributed solar. this latest procurement brings the total renewables offtake between the companies to 395mw. " by providing meaningful economic benefits to local communities including new local jobs this project will help build a stronger community a stronger energy grid as well as contribute toward our operational clean energy goals " said frank palladino vp of renewable energy strategy at walmart. this is the latest ppa secured from edpr na in march which started construction on a 100mw solar pv plant in illinois earlier this month with solar financier and operator volt energy utility. the developers secured a 15 year ppa offtake with technology giant microsoft for that project. on top of the ppa in texas with edpr na walmart has also invested in 19 solar projects under development across the us including']

retrieved_context:
scout, colgate - palmolive sign ppa for texas solar farm : scout clean energy a colorado based renewable energy developer owner and operator has signed a power purchase agreement with colgate palmolive for power from scout ' s 209 mw dc markum solar farm near waco texas. this fall scout will begin and manage project construction. markum will generate enough clean energy for the ercot grid to power approximately 32 200 homes. " virtual power purchase agreements vppas are a valuable part of our renewable energy master plan and will help us achieve our targets of net zero carbon emissions by 2040 and 100 renewable electricity across our global operations by 2030 " says ann tracy colgate palmolive chief sustainability officer in an announcement. markum is in both mclennan and bosque counties and is scheduled for delivery in 2024. the project will result in approximately 300 temporary construction jobs and three permanent full time positions when the facility begins operation. economic benefits to the community from new tax revenue over the life of the project are expected to be significant. scout clean energy is a portfolio company of brookfield renewable a publicly traded pure play renewable power platform. brookfield acquired scout at the beginning of 2023 and scout continues to execute on its significant growth plans. scout ' s legal advisor

agreement will help us power our community with an additional cost effective investment in clean and renewable geothermal energy that aligns with our dedication to environmentally responsible generation resources " said svp chief utility officer manuel pineda. in 2022 calpine signed a similar 100 mw ppa with the sacramento municipality utility district smud for 10 years. this ppa was scheduled to take effect on january 1 2023.

america signed a ppa with nestle for its under construction ganado solar plus storage project in jackson county in the south east of the state and projects such as these will help the company realise its solar power goals. the firm already manages 3. 3gw of installed wind and solar capacity in texas and plans to add another 1. 8gw of solar capacity in the coming years through projects in its pipeline. the news follows a string of new manufacturing announcements in the us such as trina solar ' s decision to build a new 5gw module manufacturing facility in the country. newfound enthusiasm for domestic manufacturing could help minimise the costs of building new solar facilities in the us although it remains to be seen if this expanded manufacturing capacity will translate to growth in power generation capacity.

mention soaring energy costs ppas are becoming an increasingly attractive option for businesses looking to secure a clean energy supply. greek energy company mytilineos recently signed two long term ppas with keppel dc reit to supply two dublin based data centres with energy generated from irish solar farms. with their high electricity demand data centres are ideal candidates for clean energy ppas. meanwhile solar developers are increasingly seeking novel and significant finance options to develop or expand solar projects. british solar renewables bsr has recently succeeded in this arena having secured its first portfolio financing thanks to a facility from commonwealth bank of australia and lloyds bank. the newly funded portfolio comprises five solar assets in the uk and australia with a combined annual solar output of 132gwh. thank you for subscribing to the newsletter. you can opt out at any time.

walmart inks solar pv ppa in texas, invests in community solar across us : retail giant walmart has signed a power purchase agreement ppa with utility edp renewables north america edpr na for a 180mw solar pv project in texas us. under a 15 year ppa walmart will offtake 162mw of the pv plant and complement existing corporate ppa between the companies on 233mw of utility scale solar and wind energy as well as 36mw of distributed solar. this latest procurement brings the total renewables offtake between the companies to 395mw. " by providing meaningful economic benefits to local communities including new local jobs this project will help build a stronger community a stronger energy grid as well as contribute toward our operational clean energy goals " said frank palladino vp of renewable energy strategy at walmart. this is the latest ppa secured from edpr na in march which started construction on a 100mw solar pv plant in illinois earlier this month with solar financier and operator volt energy utility. the developers secured a 15 year ppa offtake with technology giant microsoft for that project. on top of the ppa in texas with edpr na walmart has also invested in 19 solar projects under development across the us including

answer:
yes

answer_llm:
yes


--- Sample 3 ---
question:
What is the status of ZeroAvia's hydrogen fuel cell electric aircraft?

relevant_text:
In December, the US startup ZeroAvia announced that its retrofitted 19-seat Dornier 228 hydrogen fuel cell electric aircraft has received permission to fly from the UK Civil Aviation Authority.

relevant_text_llm:
['the wait for hydrogen fuel cell electric aircraft just got shorter : the us firm zeroavia is one step closer to bringing its zero emission electric aircraft to market powered by hydrogen fuel cells. the jet tracking social media account elonjet has drawn renewed attention to the role of private and corporate aircraft in blurting greenhouse gases into the atmosphere. the issue can be an embarrassing one for electric car makers and other businesses that portray themselves as planet savers. fortunately help is on the way. batteries and fuel cells are in a neck and neck race to nudge zero emission electric aircraft out of the workshed and onto the runway. batteries and fuel cells have already scaled up for a wide range of heavy duty ground applications including semi trucks locomotives and construction vehicles along with stationary energy storage systems. sending them up on an airplane is a different matter entirely but a flurry of activity this past fall indicates that zero emission aircraft are close at hand. in the battery area for example last fall the us startup eviation laid claim to the first public test flight of a 9 seat battery powered airplane. the company heart aerospace is also on track for commercial development of its 30 seat battery powered electric aircraft. hydrogen fuel cell aircraft have been emerging at a similar pace. in december the us startup zeroavia announced that its retro', 'expects to submit the paperwork for commercial versions of its za600 electric aircraft ranging from 9 to 19 seats by the end of 2023. the next step will be the delivery of powertrains by 2025 for which zeroavia seems well prepared. the company states that is has logged pre orders for 1 500 engines and has formed partnerships with seven aircraft manufacturers along with fuel and airport partnerships. as for why the uk is involved that \' s a good question considering that zeroavia \' s home base is far away in the state of washington. zeroavia maintains a headquarters in the uk where hydrogen fueled transportation has been catching on more quickly compared to the us. the uk and europe are also ahead of the us on on green hydrogen with an assist from a vigorous offshore wind industry. " low carbon hydrogen is our new home grown super fuel which will be vital for our energy security and to meet our legally binding commitment to achieve net zero by 2050 " explained the uk government in an update to its hydrogen strategy last summer. zeroavia \' s uk activity is funded through the 12. 3 million hyflyer ii project which builds on zeroavia \' s previous work with the uk government and other stakeholders. zeroavia has attracted interest from a list private sector investors as well. by december', 'aircraft have been emerging at a similar pace. in december the us startup zeroavia announced that its retrofitted 19 seat dornier 228 hydrogen fuel cell electric aircraft has received permission to fly from the uk civil aviation authority. the caa permission slip validates zeroavia \' s engineering and ground testing regimen opening the gate to commercialization. the dornier builds on zeroavia \' s earlier work on a six seat electric aircraft. the initial focus is on a range of 300 miles for the 19 seater. zeroavia anticipates bumping that up to 700 miles for 40 80 seats by 2027. the new caa permit sets the stage for a series of test flights to begin in january with a conventional engine on board for safety during the initial flights. " the 19 seat twin engine aircraft has been retrofitted in an engineering testbed configuration to incorporate zeroavia \' s hydrogen electric engine powering the propellor on its left wing operating alongside a single honeywell tpe 331 stock engine on the right for appropriate redundancy to allow the safe testing of the novel propulsion technology " zeroavia explained in a press release. if the test flights pan out zeroavia expects to submit the paperwork for commercial versions of its za600 electric aircraft ranging from 9 to 19', 'not to be outdone the us army also piloted beta \' s evotl last year. the army emphasized it is not ready to place an order but they did indicate a strong interest based on the compatibility of the electric aircrafts with the army \' s climate strategy. potential benefits also include " design flexibility reduced maintenance reduced acoustic signatures and less operations and support costs " according to the army. as of this writing military interest in fuel cell electric aircraft appears confined to unmanned aircraft with the navy exploring a futuristic " infinite flight " fuel cell system powered by solar energy. follow me on trainwreck twitter tinamcasey. photo hydrogen fuel cell electric aircraft courtesy of zeroavia. tina specializes in military and corporate sustainability advanced technology emerging materials biofuels and water and wastewater issues. views expressed are her own. follow her on twitter tinamcasey and spoutible. a new cargo ship will use 100 wind energy to ferry goods between europe and the us without the carbon baggage. the us state of north dakota is among those not waiting around for the green hydrogen grass to grow under their feet. large or small heat pumps are a key pathway for rapid decarbonization. in sales pitches for methanol and ammonia for maritime fuels the numbers don \' t add up and', '6th generation... eviation aircraft has just announced that the order book for the all electric alice airplane has passed a total value of 2 billion. eviation is... the us hydrogen strategy was positioned in the wrong federal department. it was put in the hands of people who deal with fossil fuels... green hydrogen for energy is an economic dead end not a replacement for fossil fuel revenues and the sooner queensland policy makers internalize that... in a recent report from rethink energy lead analyst bogdan avramuta makes the point that depending on " sustainable aviation fuel " that is kerosene... i write quite a bit about unmanned electric aircraft and the benefits both for people and the environment that they come with as they... delegates to cop27 will travel to the conference with net zero emissions rather than contributing to climate change. etihad airways has partnered with world... aviation contributes significantly to greenhouse gas emissions. in the united states commercial flights produce about 805 million metric tons of carbon dioxide per year... electric manned aircraft are under development by a number of businesses although they are still in their infancy. large planes require too much energy... xpeng got a lot of headlines this week for the first global public']

retrieved_context:
the wait for hydrogen fuel cell electric aircraft just got shorter : the us firm zeroavia is one step closer to bringing its zero emission electric aircraft to market powered by hydrogen fuel cells. the jet tracking social media account elonjet has drawn renewed attention to the role of private and corporate aircraft in blurting greenhouse gases into the atmosphere. the issue can be an embarrassing one for electric car makers and other businesses that portray themselves as planet savers. fortunately help is on the way. batteries and fuel cells are in a neck and neck race to nudge zero emission electric aircraft out of the workshed and onto the runway. batteries and fuel cells have already scaled up for a wide range of heavy duty ground applications including semi trucks locomotives and construction vehicles along with stationary energy storage systems. sending them up on an airplane is a different matter entirely but a flurry of activity this past fall indicates that zero emission aircraft are close at hand. in the battery area for example last fall the us startup eviation laid claim to the first public test flight of a 9 seat battery powered airplane. the company heart aerospace is also on track for commercial development of its 30 seat battery powered electric aircraft. hydrogen fuel cell aircraft have been emerging at a similar pace. in december the us startup zeroavia announced that its retro

expects to submit the paperwork for commercial versions of its za600 electric aircraft ranging from 9 to 19 seats by the end of 2023. the next step will be the delivery of powertrains by 2025 for which zeroavia seems well prepared. the company states that is has logged pre orders for 1 500 engines and has formed partnerships with seven aircraft manufacturers along with fuel and airport partnerships. as for why the uk is involved that ' s a good question considering that zeroavia ' s home base is far away in the state of washington. zeroavia maintains a headquarters in the uk where hydrogen fueled transportation has been catching on more quickly compared to the us. the uk and europe are also ahead of the us on on green hydrogen with an assist from a vigorous offshore wind industry. " low carbon hydrogen is our new home grown super fuel which will be vital for our energy security and to meet our legally binding commitment to achieve net zero by 2050 " explained the uk government in an update to its hydrogen strategy last summer. zeroavia ' s uk activity is funded through the 12. 3 million hyflyer ii project which builds on zeroavia ' s previous work with the uk government and other stakeholders. zeroavia has attracted interest from a list private sector investors as well. by december

aircraft have been emerging at a similar pace. in december the us startup zeroavia announced that its retrofitted 19 seat dornier 228 hydrogen fuel cell electric aircraft has received permission to fly from the uk civil aviation authority. the caa permission slip validates zeroavia ' s engineering and ground testing regimen opening the gate to commercialization. the dornier builds on zeroavia ' s earlier work on a six seat electric aircraft. the initial focus is on a range of 300 miles for the 19 seater. zeroavia anticipates bumping that up to 700 miles for 40 80 seats by 2027. the new caa permit sets the stage for a series of test flights to begin in january with a conventional engine on board for safety during the initial flights. " the 19 seat twin engine aircraft has been retrofitted in an engineering testbed configuration to incorporate zeroavia ' s hydrogen electric engine powering the propellor on its left wing operating alongside a single honeywell tpe 331 stock engine on the right for appropriate redundancy to allow the safe testing of the novel propulsion technology " zeroavia explained in a press release. if the test flights pan out zeroavia expects to submit the paperwork for commercial versions of its za600 electric aircraft ranging from 9 to 19

not to be outdone the us army also piloted beta ' s evotl last year. the army emphasized it is not ready to place an order but they did indicate a strong interest based on the compatibility of the electric aircrafts with the army ' s climate strategy. potential benefits also include " design flexibility reduced maintenance reduced acoustic signatures and less operations and support costs " according to the army. as of this writing military interest in fuel cell electric aircraft appears confined to unmanned aircraft with the navy exploring a futuristic " infinite flight " fuel cell system powered by solar energy. follow me on trainwreck twitter tinamcasey. photo hydrogen fuel cell electric aircraft courtesy of zeroavia. tina specializes in military and corporate sustainability advanced technology emerging materials biofuels and water and wastewater issues. views expressed are her own. follow her on twitter tinamcasey and spoutible. a new cargo ship will use 100 wind energy to ferry goods between europe and the us without the carbon baggage. the us state of north dakota is among those not waiting around for the green hydrogen grass to grow under their feet. large or small heat pumps are a key pathway for rapid decarbonization. in sales pitches for methanol and ammonia for maritime fuels the numbers don ' t add up and

6th generation... eviation aircraft has just announced that the order book for the all electric alice airplane has passed a total value of 2 billion. eviation is... the us hydrogen strategy was positioned in the wrong federal department. it was put in the hands of people who deal with fossil fuels... green hydrogen for energy is an economic dead end not a replacement for fossil fuel revenues and the sooner queensland policy makers internalize that... in a recent report from rethink energy lead analyst bogdan avramuta makes the point that depending on " sustainable aviation fuel " that is kerosene... i write quite a bit about unmanned electric aircraft and the benefits both for people and the environment that they come with as they... delegates to cop27 will travel to the conference with net zero emissions rather than contributing to climate change. etihad airways has partnered with world... aviation contributes significantly to greenhouse gas emissions. in the united states commercial flights produce about 805 million metric tons of carbon dioxide per year... electric manned aircraft are under development by a number of businesses although they are still in their infancy. large planes require too much energy... xpeng got a lot of headlines this week for the first global public

answer:
ZeroAvia's hydrogen fuel cell electric aircraft, a retrofitted 19-seat Dornier 228, has received permission to fly from the UK Civil Aviation Authority.

answer_llm:
ZeroAvia's hydrogen fuel cell electric aircraft has received permission to fly from the UK Civil Aviation Authority and is set to begin test flights in January.

	id	title	date	author	content	domain	url
0	93320	XPeng Delivered ~100,000 Vehicles In 2021	2022-01-02	NaN	['Chinese automotive startup XPeng has shown o...	cleantechnica	https://cleantechnica.com/2022/01/02/xpeng-del...
1	93321	Green Hydrogen: Drop In Bucket Or Big Splash?	2022-01-02	NaN	['Sinopec has laid plans to build the largest ...	cleantechnica	https://cleantechnica.com/2022/01/02/its-a-gre...
2	98159	World’ s largest floating PV plant goes online...	2022-01-03	NaN	['Huaneng Power International has switched on ...	pv-magazine	https://www.pv-magazine.com/2022/01/03/worlds-...
3	98158	Iran wants to deploy 10 GW of renewables over ...	2022-01-03	NaN	['According to the Iranian authorities, there ...	pv-magazine	https://www.pv-magazine.com/2022/01/03/iran-wa...
4	31128	Eastern Interconnection Power Grid Said ‘ Bein...	2022-01-03	NaN	['Sign in to get the best natural gas news and...	naturalgasintel	https://www.naturalgasintel.com/eastern-interc...

	title	date	content	domain	url
count	20111	20111	20111	20111	20111
unique	20016	979	20068	25	20111
top	About David J. Cross	2023-06-13	["By the end of 2022, 37% of single-family homes a cross the five islands Hawaiian Electric serves ha d rooftop solar, Oahu's first grid-scale solar and storage project was online, and the state's last remaining coal plant was retired. That's the kind ...	energy-xprt	https://cleantechnica.com/2022/01/02/xpeng-deliver ed-100000-vehicles-in-2021/
freq	13	1812	3	4181	1

	id	title	date	content	domain	url
1721	21198	About David J. Cross	2022-05-26 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=6
6940	22486	About David J. Cross	2023-05-04 00:00:00	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=4
6941	22487	About David J. Cross	2023-05-04 00:00:00	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=2
6942	22488	About David J. Cross	2023-05-04 00:00:00	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=11
7107	22482	About David J. Cross	2023-05-04 00:00:00	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	azocleantech	https://www.azocleantech.com/authors/david-cross
7110	22485	About David J. Cross	2023-05-04 00:00:00	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=3
13074	22798	About David J. Cross	2023-08-19 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=13
13076	22800	About David J. Cross	2023-08-19 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=12
13081	22807	About David J. Cross	2023-08-19 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=5
13098	22814	About David J. Cross	2023-08-20 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=9
13104	22809	About David J. Cross	2023-08-20 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=8
13105	22808	About David J. Cross	2023-08-20 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=7
13217	22852	About David J. Cross	2023-08-24 00:00:00	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	azocleantech	https://www.azocleantech.com/authors/david-cross?p age=10

	date	count
507	2023-06-13	1812
508	2023-06-14	1493
468	2023-05-04	430
509	2023-06-15	340
464	2023-04-30	111
715	2024-01-11	53
622	2023-10-07	53
653	2023-11-08	49
618	2023-10-03	49
579	2023-08-25	46

	domain	count
0	energy-xprt	1760
1	cleantechnica	20
2	decarbxpo	12
3	pv-magazine	8
4	azocleantech	3
5	solarpowerworldonline	3
6	ecofriend	1
7	energyintel	1
8	pv-tech	1
9	solarindustrymag	1
10	solarpowerportal.co	1
11	thinkgeoenergy	1

	title	count
0	Energy Storage Companies and Suppliers	6
1	Waste-to-Energy Companies and Suppliers	4
2	Incineration Companies and Suppliers	2
3	Solar Data ( Solar Energy) Equipment	1
4	Solar Energy Articles - page 2	1
...	...	...
1746	High Voltage Electricity ( Power Distribution)...	1
1747	High Voltage Cables ( Power Distribution) News	1
1748	High Voltage ( Power Distribution) News	1
1749	High Temperature Batteries Companies and Suppl...	1
1750	zinc alkaline battery ( Battery Energy Storage	1

	content	content_cleaned
1721	['We use cookies to enhance your experience. By co ntinuing to browse this site you agree to our use of cookies. More info.', 'B.A ( Hons) Comparative Literary Studies, M.A. Modern Cultures', 'David is an academic researcher and interdisciplinary arti ...	'. More info.', 'B.A Hons Comparative Literary St udies, M.A. Modern Cultures', 'David is an academi c researcher and interdisciplinary artist primaril y working with text, video, sound, and new technol ogies. His current research and practice is focuse ...
6940	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	"By clicking Allow All '' you agree to the storin g of cookies on your device to enhance site naviga tion, analyse site usage and support us in providi ng free open access scientific content. More info. ", 'B.A Hons Comparative Literary Studies, M.A. M ...
6941	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	"By clicking Allow All '' you agree to the storin g of cookies on your device to enhance site naviga tion, analyse site usage and support us in providi ng free open access scientific content. More info. ", 'B.A Hons Comparative Literary Studies, M.A. M ...
6942	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	"By clicking Allow All '' you agree to the storin g of cookies on your device to enhance site naviga tion, analyse site usage and support us in providi ng free open access scientific content. More info. ", 'B.A Hons Comparative Literary Studies, M.A. M ...
7107	["By clicking `` Allow All '' you agree to the sto ring of cookies on your device to enhance site nav igation, analyse site usage and support us in prov iding free open access scientific content. More in fo.", 'B.A ( Hons) Comparative Literary Studies, M ...	"By clicking Allow All '' you agree to the storin g of cookies on your device to enhance site naviga tion, analyse site usage and support us in providi ng free open access scientific content. More info. ", 'B.A Hons Comparative Literary Studies, M.A. M ...

	id	title	date	content	domain	url	language	content_cleaned
0	93320	XPeng Delivered ~100,000 Vehicles In 2021	2022-01-02 00:00:00	Chinese automotive startup XPeng has shown one of the most dramatic auto production ramp-ups in hist ory and the good news is it only produces 100% sma rt electric vehicles ( EVs). At a mere 7 years of age and just a few years after launching its first ...	cleantechnica	https://cleantechnica.com/2022/01/02/xpeng-deliver ed-100000-vehicles-in-2021/	en	Chinese automotive startup XPeng has shown one of the most dramatic auto production ramp ups in hist ory and the good news is it only produces 100 smar t electric vehicles EVs. At a mere 7 years of age and just a few years after launching its first ve ...
1	93321	Green Hydrogen: Drop In Bucket Or Big Splash?	2022-01-02 00:00:00	Sinopec has laid plans to build the largest green hydrogen production facility in the world but inte rest in fossil-sourced hydrogen continues apace. S kepticism abounds over the green hydrogen field an d yet leading players on the global energy stage s ...	cleantechnica	https://cleantechnica.com/2022/01/02/its-a-green-h ydrogen-drop-in-the-bucket-but-it-could-still-make -a-splash/	en	Sinopec has laid plans to build the largest green hydrogen production facility in the world but inte rest in fossil sourced hydrogen continues apace. S kepticism abounds over the green hydrogen field an d yet leading players on the global energy stage s ...
2	98159	World’ s largest floating PV plant goes online in China – pv magazine International	2022-01-03 00:00:00	Huaneng Power International has switched on a 320 MW floating PV array in China’ s Shandong province . It deployed the plant in two phases on a reservo ir near its 2.65 GW Dezhou thermal power station. Huaneng Power International ( HPI) has completed t ...	pv-magazine	https://www.pv-magazine.com/2022/01/03/worlds-larg est-floating-pv-plant-goes-online-in-china/	en	Huaneng Power International has switched on a 320 MW floating PV array in China' s Shandong province . It deployed the plant in two phases on a reservo ir near its 2.65 GW Dezhou thermal power station. Huaneng Power International HPI has completed the ...

	title	content	domain	url	language	content_cleaned
count	20106	20106	20106	20106	20106	20106
unique	20011	20063	25	20106	1	19988
top	About David J. Cross	By the end of 2022 37% of single-family homes acro ss the five islands Hawaiian Electric serves had r ooftop solar Oahu's first grid-scale solar and sto rage project was online and the state's last remai ning coal plant was retired. That's the kind of pr ...	energy-xprt	https://cleantechnica.com/2022/01/02/xpeng-deliver ed-100000-vehicles-in-2021/	en	Energy Human development Solar Energy Wind Power 01 20 September 2014 Paderborn Germany. The scie ntific engineering program focus on solar energy w ind power human development and energy transition as well as lectures on the basic of renewable ene ...
freq	13	3	4181	1	20106	10

DB-Name	TextSplitter	Chunk Size / Overlap	Ähnlichkeitsmetrik	Erwartete Auswirkungen
`recchar_800_cosine`	`RecursiveCharacterTextSplitter`	800 Zeichen / 100	Cosine	Gute Balance zwischen Genauigkeit und Effizienz; geringer Ressourcenverbrauch.
`recchar_800_l2`	`RecursiveCharacterTextSplitter`	800 Zeichen / 100	L2 (Euklidisch)	Potenziell bessere Treffergenauigkeit; höherer Rechen- und Speicherbedarf.
`recchar_800_ip`	`RecursiveCharacterTextSplitter`	800 Zeichen / 100	IP (Inneres Produkt)	Abhängig vom Textinhalt; bei passenden Daten eventuell bessere Relevanz.
`senttrans_200_cosine`	`SentenceTransformersTokenTextSplitter`	200 Token / 20	Cosine	Höhere Präzision bei modernen Modellen; Chunkgrenzen wirken teils unnatürlich.
`senttrans_200_l2`	`SentenceTransformersTokenTextSplitter`	200 Token / 20	L2 (Euklidisch)	Sehr genaue Ergebnisse möglich; deutlich erhöhter Ressourcenbedarf.
`senttrans_200_ip`	`SentenceTransformersTokenTextSplitter`	200 Token / 20	IP (Inneres Produkt)	Uneinheitliche Resultate; sinnvoll bei bestimmten Textarten oder Spezialanwendungen.

Emb. Modell	Distanzmetrik	Splitter	Chunks	Faithfulness	Answer Relevance	Context Precision	Context Recall	Answer Correctness	MMR	Precision@k	Recall@k	Similarity to Gold
3-large	cosine	recchar	800	0.88	0.49	0.88	0.88	0.66	0.95	0.81	1.00	0.778
ada	cosine	recchar	800	0.82	0.84	0.83	0.91	0.73	1.00	1.00	1.00	0.937
3-large	IP	recchar	800	0.88	0.49	0.88	0.88	0.62	0.95	0.81	1.00	0.749
ada	IP	recchar	800	0.82	0.84	0.81	0.88	0.74	1.00	1.00	1.00	0.918
3-large	l2	recchar	800	0.88	0.49	0.88	0.88	0.65	0.95	0.81	1.00	0.808
ada	l2	recchar	800	0.84	0.84	0.83	0.88	0.76	1.00	1.00	1.00	0.918
3-large	cosine	senttrans	200	0.88	0.49	0.85	0.95	0.64	0.91	0.77	0.95	0.825
ada	cosine	senttrans	200	0.92	0.84	0.92	0.95	0.71	1.00	1.00	1.00	0.946
3-large	IP	senttrans	200	0.88	0.49	0.86	0.95	0.63	0.91	0.77	0.95	0.834
ada	IP	senttrans	200	0.86	0.84	0.92	0.95	0.70	1.00	1.00	1.00	0.943
3-large	l2	senttrans	200	0.88	0.48	0.84	0.95	0.68	0.91	0.77	0.95	0.835
ada	l2	senttrans	200	0.92	0.84	0.92	0.95	0.75	1.00	1.00	1.00	0.942

Retriever	Prompt	k	Faithfulness	Answer Relevance	Context Precision	Context Recall	Answer Correctness	MMR	Precision@k	Recall@k	Similarity to Gold
similarity	simple	5	0.86	0.84	0.92	0.95	0.70	1.00	1.00	1.00	0.943
similarity	cot	5	0.92	0.84	0.89	0.95	0.72	1.00	1.00	1.00	0.846
similarity	simple	3	0.90	0.84	0.99	0.95	0.68	1.00	0.60	1.00	0.933
similarity	cot	3	0.82	0.84	0.93	0.89	0.71	1.00	0.60	1.00	0.844
mmr, fetch_k=20	simple	5	0.75	0.84	0.98	0.87	0.75	1.00	1.00	1.00	0.899
mmr, fetch_k=20	cot	5	0.75	0.84	0.96	0.87	0.73	1.00	1.00	1.00	0.837
mmr, fetch_k=20	simple	7	0.73	0.84	0.97	0.88	0.71	1.00	1.00	1.00	0.901
mmr, fetch_k=20	cot	7	0.73	0.84	0.96	0.88	0.76	1.00	1.00	1.00	0.836

	NaN/None (%)	0-Werte (%)
id	0.0	0.0
title	0.0	0.0
date	0.0	0.0
author	100.0	0.0
content	0.0	0.0
domain	0.0	0.0
url	0.0	0.0

NPR MC1 RAG¶

Libraries¶

Inhaltsverzeichnis¶

Kurzbeschreibung:¶

Hinweis:¶

Sources¶

EDA¶

Daten kennen lernen¶

Daten Laden¶

Datenbeschreibung¶

Untersuchen der ersten Erkenntnisse¶

Artikel pro Domain¶

Analyse: Anzahl Artikel pro Domain¶

Top-Domains mit hoher Artikelanzahl¶

Verteilungsmuster und Interpretation¶

Mögliche Ursachen für die ungleiche Verteilung¶

Anzahl Artikel zu Top-Titel¶

Anzahl pro Domain mit dem Titel 'About David J. Cross'¶

Anzahl Artikel pro Top-Datum¶

Anzahl pro Domain zum Datum vom 13.06.2023¶

Duplikate beim Artikel?¶

Welches sind die meist benutzten Worte?¶

Sprache überprüfen¶

Text Bereinigen¶

Klasse TextCleaner¶

Duplikate entfernen¶

Titel und Inhalt-Längen¶

Gesamten Text bereinigen¶

Wordcloud 2.0¶

Titel an den Content anfügen¶

Datensatz mit 1000 Zeilen¶

Chunking¶

Tokenisierung¶

Embeddings¶

Chroma Daten¶

Übersicht der Konfigurationen¶

Detaillierte Parameterbeschreibung¶

1. Embedding-Modell¶

2. TextSplitter¶

3. Ähnlichkeitsmetriken (hnsw:space)¶

4. Konstant gehaltene Parameter¶

Fazit¶

Retrieval-Modul (Suche & Ranking) & Generator-Modul (Antwortgenerierung)¶

Ablauf der RetrievalQA-Evaluations-Chain¶

Varianten im Retrieval¶

Weitere Parameter:¶

Output:¶

Evaluator-Modul (Qualitätsprüfung & Optimierung)¶

Evaluierte Metriken¶

Durchführung des Experiments¶

Evaluierung Embeddings¶

Embedding Evaluation: Übersicht der Mittelwerte¶

Evaluierung Retrieval¶

Retriever- und Prompt-Kombinationen: Übersicht der Mittelwerte¶

Evaluierung Temperatur¶

Temperature Evaluation: Übersicht der Mittelwerte¶

Manuell plausability Check¶

Fazit¶

Empfohlene ausgewogene Systemkonfiguration:¶

Weitere Überlegungen¶

Klasse `TextCleaner`¶

3. Ähnlichkeitsmetriken (`hnsw:space`)¶